import re
import sqlite3
import os
import logging
from django.http import HttpResponse
from django.conf import settings
import json
import datetime
from xml.dom.minidom import Document
import time
from concurrent.futures import ThreadPoolExecutor,as_completed,ProcessPoolExecutor
import random

oozie_home='/export/datamarket/oozie/oozie-4.3.1/bin/'
hadoop_home='/export/datamarket/hadoop/bin/'
logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%a, %d %b %Y %H:%M:%S')
execShellDir='/export/datamarket/OozieProperties/oozie_project/oozie_shell/upload'
def modifyBasic(request):


	if request.method == 'POST':
		try:	
			received_json_data=json.loads(request.body.decode().replace("'", "\""))
			logging.info("\n\n\n\t"+str(received_json_data)+"\n\n")
			jobid=received_json_data['jobid']
			oozieMission=received_json_data['oozieMission'].strip()
			mission_val=received_json_data['oozieMission'].strip()
			oozieTime=received_json_data['oozieTime'].strip()
			time_val=received_json_data['oozieTime'].strip()
			erpName=received_json_data['erpName'].strip()
			erpName_val=received_json_data['erpName'].strip()
			erpPassword=received_json_data['erpPassword'].strip()
			erpPassword_val=received_json_data['erpPassword'].strip()
			shellID=received_json_data['shellID'].strip()
			
			result={'result':oozieMission+'任务修改成功','code':0}
			result2={'result':oozieMission+'任务已经存在','code':1}

			today='2018-11-10'

			db_path = os.path.normpath('/export/datamarket/OozieProperties/oozie_project/db.sqlite3')
			db = sqlite3.connect(db_path) 
			# 通过cursor创建游标
			cursor = db.cursor()
			sql_jobid="select mission_val from oozie_configuration where jobid ={}".format(jobid)
			cursor.execute(sql_jobid)
			mission_original=cursor.fetchall()
			mission_original=mission_original[0][0]
			sql_update="update oozie_configuration set mission_val='{}',time_val='{}',erpName_val='{}',erpPassword_val='{}' where jobid='{}'".format(mission_val,time_val,erpName_val,erpPassword_val,jobid)
			cursor.execute(sql_update)
			logging.info("修改sqlite中jobid为"+jobid+"的信息")
			#修改对应jobid的mission的目录
			os.system("mv /export/datamarket/OozieExec/"+today+"/"+mission_original+" /export/datamarket/OozieExec/"+today+"/"+mission_val)
			#杀死原来任务
			randomNum=os.popen('echo -n `date +%s%N`').read()
			jobIDinfoFile='jobIDinfo'+str(randomNum)+str(random.uniform(1, 24))
			jobidFile='jobid'+str(randomNum)+str(random.uniform(1, 24))
			os.system(oozie_home+"oozie jobs -localtime -filter 'name={}' -jobtype coordinator -len 1 > {}".format(mission_original,jobIDinfoFile))
			os.system("sed -i '1d' {}".format(jobIDinfoFile))
			os.system("cat {}".format(jobIDinfoFile)+" |awk -F ' ' '{print$1}' >"+jobidFile)
			os.system("sed -i '1d' {}".format(jobidFile))
			os.system("sed -i '2d' {}".format(jobidFile))
			coordinatorJobid=os.popen("cat {}".format(jobidFile)).read().strip()
			os.remove("{}".format(jobIDinfoFile))
			os.remove("{}".format(jobidFile))
			os.system(oozie_home+"oozie job -kill {}".format(coordinatorJobid))
			#需要执行的脚本
			sql_shell=  "select * from oozie_shell where shellID='{}'".format(shellID) 
			cursor.execute(sql_shell)
			shellID=cursor.fetchone()
			oozieShellName=shellID[1]
			sql_update_shellName="update oozie_configuration set shellName='{}' where mission_val='{}'".format(oozieShellName,mission_val)
			cursor.execute(sql_update_shellName)
			logging.info("更新sqlite中任务"+mission_val+"对应脚本"+oozieShellName)
			sql_update_shellID="update oozie_configuration set shellID='{}' where mission_val='{}'".format(shellID[0],mission_val)              
			logging.info(sql_update_shellID)
			cursor.execute(sql_update_shellID)
			logging.info("更新sqlite中任务"+mission_val+"对应脚本ID"+str(shellID[0]))
			cursor.close()
			db.commit()
			db.close()
			#oozie提交任务
			oozieShell=execShellDir+"/"+oozieShellName
			run(mission_val,oozieShell)
			logging.info(result)
			return HttpResponse(json.dumps(result,ensure_ascii=False), content_type="application/json;charset=utf-8")
				
			
		except:
			logging.info('\n\n\n\t输入格式有误\n\n')
			resultError={'result':'请检查格式','code':'1'}
			return HttpResponse(json.dumps(resultError,ensure_ascii=False), content_type="application/json;charset=utf-8")
	else:
		pass
def main(todayfile,today,mission_val,time_val,erpName_val,erpPassword_val):
	today=today
	todayfile=todayfile
	mission_val=mission_val
	time_val=time_val
	erpName_val=erpName_val
	erpPassword_val=erpPassword_val
	fileList=[]
	jobproperties='job.properties'
	##oozie其他固定配置文件所在目录
	ooziePorperties='/export/datamarket/OoziePorperties/'
	#多action需要的
	oozieWorkflowAction='/export/datamarket/OozieScript/action'
	#生成workflow.xml文件
	def workflowCreate(mission_val,name,end,allFile):
		
		doc = Document()
		workflow = doc.createElement('workflow-app')
		workflow.setAttribute('xmlns','uri:oozie:workflow:0.4')
		workflow.setAttribute('name','%s'%mission_val)
		doc.appendChild(workflow)
		
		start = doc.createElement("start")
		start.setAttribute('to','%s'%name)
		workflow.appendChild(start)
		
		action=doc.createElement("action")
		action.setAttribute('name','%s'%name)
		action.setAttribute('retry-max','3')
		action.setAttribute('retry-interval','1')
		workflow.appendChild(action)
		
		shell=doc.createElement("shell")
		shell.setAttribute('xmlns','uri:oozie:shell-action:0.2')
		action.appendChild(shell)
		
		jobtracker=doc.createElement("job-tracker")
		shell.appendChild(jobtracker)
		jobtracker_name=doc.createTextNode("${jobTracker}")
		jobtracker.appendChild(jobtracker_name)
		
		namenode=doc.createElement("name-node")
		shell.appendChild(namenode)
		namenode_name=doc.createTextNode("${nameNode}")
		namenode.appendChild(namenode_name)
		
		prepare=doc.createElement("prepare")
		shell.appendChild(prepare)
		
		deletepath=doc.createElement("delete")
		deletepath.setAttribute('path','${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/')
		prepare.appendChild(deletepath)
		mkdirpath=doc.createElement("mkdir")
		mkdirpath.setAttribute('path','${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/')
		prepare.appendChild(mkdirpath)
		
		configuration=doc.createElement("configuration")
		shell.appendChild(configuration)
		property=doc.createElement("property")
		configuration.appendChild(property)
		property_tag_name=doc.createElement("name")
		property.appendChild(property_tag_name)
		property_tag_name_name=doc.createTextNode("mapred.job.queue.name")
		property_tag_name.appendChild(property_tag_name_name)
		property_tag_value=doc.createElement("value")
		property.appendChild(property_tag_value)
		property_tag_value_name=doc.createTextNode("${queueName}")
		property_tag_value.appendChild(property_tag_value_name)
		
		property2=doc.createElement("property")
		configuration.appendChild(property2)
		property2_tag_name=doc.createElement("name")
		property2.appendChild(property2_tag_name)
		property2_tag_name_name=doc.createTextNode("oozie.launcher.mapreduce.map.memory.mb")
		property2_tag_name.appendChild(property2_tag_name_name)
		property2_tag_value=doc.createElement("value")
		property2.appendChild(property2_tag_value)
		property2_tag_value_name=doc.createTextNode("8192")
		property2_tag_value.appendChild(property2_tag_value_name)
		
		property3=doc.createElement("property")
		configuration.appendChild(property3)
		property3_tag_name=doc.createElement("name")
		property3.appendChild(property3_tag_name)
		property3_tag_name_name=doc.createTextNode("oozie.launcher.mapreduce.reduce.memory.mb")
		property3_tag_name.appendChild(property3_tag_name_name)
		property3_tag_value=doc.createElement("value")
		property3.appendChild(property3_tag_value)
		property3_tag_value_name=doc.createTextNode("8192")
		property3_tag_value.appendChild(property3_tag_value_name)
		
		execExec=doc.createElement("exec")
		shell.appendChild(execExec)
		execExec_name=doc.createTextNode("%s.sh"%name)
		execExec.appendChild(execExec_name)
		
		file=doc.createElement("file")
		shell.appendChild(file)
		file_name=doc.createTextNode("${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/%s.sh"%name)
		file.appendChild(file_name)
		
		ok=doc.createElement("ok")
		ok.setAttribute('to','%s'%end)
		action.appendChild(ok)
		error=doc.createElement("error")
		error.setAttribute('to','fail')
		action.appendChild(error)
		
		
		
		
		kill=doc.createElement("kill")
		kill.setAttribute("name","fail")
		workflow.appendChild(kill)
		
		message=doc.createElement("message")
		kill.appendChild(message)
		message_name=doc.createTextNode("Shell failed, error message[${wf:errorMessage(wf:lastErrorNode())}]")
		message.appendChild(message_name)
		
		end=doc.createElement("end")
		end.setAttribute("name","end")
		workflow.appendChild(end)
		
		with open(allFile+'/workflow.xml','w') as f:
			f.write(doc.toprettyxml(indent="  "))
			
		
	#生成coordinator.xml文件
	def coordinator(mission_val,allFile):
		doc = Document()
		coordinatorapp=doc.createElement("coordinator-app")
		coordinatorapp.setAttribute("xmlns","uri:oozie:coordinator:0.4")
		coordinatorapp.setAttribute("timezone","Asia/Shanghai")
		coordinatorapp.setAttribute("start","${start}")
		coordinatorapp.setAttribute("name","{}".format(mission_val))
		coordinatorapp.setAttribute("frequency","${frequency}")
		coordinatorapp.setAttribute("end","${end}")
		doc.appendChild(coordinatorapp)
		
		action=doc.createElement("action")
		coordinatorapp.appendChild(action)
		
		workflow=doc.createElement("workflow")
		action.appendChild(workflow)
		
		apppath=doc.createElement("app-path")
		workflow.appendChild(apppath)
		apppath_name=doc.createTextNode("${workflowAppUri}")
		apppath.appendChild(apppath_name)
		
		configuration=doc.createElement("configuration")
		workflow.appendChild(configuration)
		
		property=doc.createElement("property")
		configuration.appendChild(property)
		property_tag_name=doc.createElement("name")
		property.appendChild(property_tag_name)
		property_tag_name_name=doc.createTextNode("jobTracker")
		property_tag_name.appendChild(property_tag_name_name)
		property_tag_value=doc.createElement("value")
		property.appendChild(property_tag_value)
		property_tag_value_name=doc.createTextNode("${jobTracker}")
		property_tag_value.appendChild(property_tag_value_name)
		
		property2=doc.createElement("property")
		configuration.appendChild(property2)
		property2_tag_name=doc.createElement("name")
		property2.appendChild(property2_tag_name)
		property2_tag_name_name=doc.createTextNode("nameNode")
		property2_tag_name.appendChild(property2_tag_name_name)
		property2_tag_value=doc.createElement("value")
		property2.appendChild(property2_tag_value)
		property2_tag_value_name=doc.createTextNode("${nameNode}")
		property2_tag_value.appendChild(property2_tag_value_name)
		
		property3=doc.createElement("property")
		configuration.appendChild(property3)
		property3_tag_name=doc.createElement("name")
		property3.appendChild(property3_tag_name)
		property3_tag_name_name=doc.createTextNode("queueName")
		property3_tag_name.appendChild(property3_tag_name_name)
		property3_tag_value=doc.createElement("value")
		property3.appendChild(property3_tag_value)
		property3_tag_value_name=doc.createTextNode("${queueName}")
		property3_tag_value.appendChild(property3_tag_value_name)

		with open(allFile+'/coordinator.xml','w') as f:
			f.write(doc.toprettyxml(indent="  "))
		

	#删除文件
	def deleteFile(deleteFile):
		deleteFile=deleteFile
		if os.path.exists(deleteFile):
			os.remove(deleteFile)
		else:
			pass
	
	allFile=todayfile
	if (os.path.isdir(allFile)):
		
		#dirList=execPath.split('/')[-1]

		#allfileList.append(allFile)
		#allDir=allFile
		#os.system("rm "+allFile+"/ceshiduo.properties")
		execNum=1
		for file in os.listdir(allFile):
			if file.endswith('.sh'):
				fileList.append(file)
				#os.system("mkdir "+execPath+file.split(".")[0])
				#print("mkdir "+execPath+file.split(".")[0])
				#os.system("rm "+allFile+"/"+jobproperties)
				#定时执行的job.properties
				deletefile=allFile+"/"+jobproperties
				deleteFile(deletefile)
				#立即执行的job.properties
				deletefile=allFile+"/jobImmediately.properties"
				deleteFile(deletefile)
				execDir=allFile.split("/")[-1]
				execDir='execDir='+execDir
				with open(allFile+'/'+jobproperties,'a') as f:
					f.write(execDir+"\n")
				with open(allFile+'/jobImmediately.properties','a') as f:
					f.write(execDir+"\n")
	
				execNum+=1
			#elif file.startswith('time'):
			#	print("配置文件是："+file)
			else:
				pass
		
		execNum2=1
		nameList=[]
		#os.system("rm "+allFile+"/workflow.xml")
		workflowxml=allFile+"/workflow.xml"
		if os.path.exists(workflowxml):
			os.remove(workflowxml)
		else:
			pass
	
		#os.system("rm "+allFile+"/coordinator.xml")
		coordinatorxml=allFile+"/coordinator.xml"
		deleteFile(coordinatorxml)
		for file in os.listdir(allFile):
			if file.endswith('.sh'):
				fileList.append(file)
				execDir0=allFile.split("/")[-1]
				execDir='execDir='+execDir0
				name=file.split(".")[0]
				#print('name'+str(execNum2)+'='+name)
				actionname='EXEC'+str(execNum2)
				actionExecFile=actionname+'.sh'
				strName='EXEC'+str(execNum2)+'='+name+'.sh'
				
				nameList.append(name)
				nameList.sort()
				if actionname=='EXEC1':
					end='end'
					
					workflowCreate(mission_val,name,end,allFile)
					###检测workflow的有效性
					os.system(oozie_home+"oozie validate "+allFile+"/workflow.xml")
					#coordinator(allFile)
					coordinator(mission_val,allFile)
					
					####################################################时间
					#os.system("cp "+ooziePorperties+"/time.properties "+allFile+"/")
					#os.system("cat "+allFile+"/time.properties >> "+allFile+"/"+jobproperties)
					#print(jobproperties)
					#print("os.system(\"cat \"+allFile+\"/time.properties >> \"+jobproperties)")						

				else:
					for end in nameList[1:]:
						nameList2=['end']
						endList =nameList[1:]+nameList2
						
							
					name=nameList[0]
					name="'"+name+"'"
					#print(name)
					os.chdir(allFile+"/")
					#os.system("if [ -f "action1 action2 action3 action4 action5 workflowCreateAction.py" ];then rm action1 action2 action3 action4 action5 workflowCreateAction.py fi")
					action1='action1'
					deleteFile(action1)
					action2='action2'
					deleteFile(action2)
					action3='action3'
					deleteFile(action3)
					action4='action4'
					deleteFile(action4)
					action5='action5'
					deleteFile(action5)
					deletefile='workflowCreateAction.py'
					deleteFile(deletefile)
					os.system("cp "+oozieWorkflowAction+"/action1 .")	
					os.system("cp "+oozieWorkflowAction+"/action3 .")	
					os.system("cp "+oozieWorkflowAction+"/action5 .")	
					with open(allFile+'/action2','w') as f:
						f.write('name='+name+"\n")
					#workflow(name,end)
					nameend=dict(zip(nameList,endList))
					#print(nameend)
					#keys.sort() 
					with open(allFile+'/actionname.txt','w') as f:
						for key in sorted(nameend.keys()):
							#sorted(nameend.items(), key=lambda d:d[1],reverse=True)
							f.write('workflowCreate(\'%s\',\'%s\')\n'%(key,nameend[key]))
					lineList=[]
					with open(allFile+'/actionname.txt','r') as r:
						for line in r.readlines():
							#print(line)
							#line=line.split('\n')
							lineList.append(line)
						lineSort=lineList[:]
						lineSort.sort()
						#print(lineSort)
						lineSortStr='\n'.join(lineSort)
						#print(lineSortStr)
					#workflowCreateStr='workflowCreateStr'.replace('workflowCreateStr',lineSortStr)
					with open(allFile+'/action4','w') as f:
						f.write(lineSortStr)
					os.system("cat "+allFile+"/action1 action2 action3 action4 action5>> workflowCreateAction.py")
					os.system("chmod +x "+allFile+"/workflowCreateAction.py")
					os.system("/usr/local/bin/python3 "+allFile+"/workflowCreateAction.py")
	
					####检测workflow的有效性
					os.system(oozie_home+"oozie validate "+allFile+"/workflow.xml")
					#coordinator(allFile)
					coordinator(mission_val,allFile)
				execNum2+=1
				
			else:
				pass
				#print('NO execfile !')
		print("共执行："+str(execNum2-1)+"个.sh文件")
		
		with open(allFile+'/'+jobproperties,'a') as f:
			###定时执行
			f.write("today="+today+"\nnameNode=hdfs://host-11-3-68-6:9000\njobTracker=host-11-3-68-6:8130\nqueueName=default\noozie.coord.application.path=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}\nworkflowAppUri=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}\noutputDir=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/\n")
		with open(allFile+'/jobImmediately.properties','a') as f:
			###只执行workflow一次
			f.write("today="+today+"\nnameNode=hdfs://host-11-3-68-6:9000\njobTracker=host-11-3-68-6:8130\nqueueName=default\noozie.wf.application.path=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}\noutputDir=${nameNode}/user/root/oozie-apps/ceshi/${today}/${execDir}/outputdir/\n")
		####################################################时间
		#os.system("cp "+ooziePorperties+"/time.properties "+allFile+"/")
		#os.system("cat "+allFile+"/time.properties >> "+allFile+"/"+jobproperties)
		####################################################时间
		#specificTimeDate=datetime.datetime.now().strftime('%Y-%m-%d')
		os.chdir(allFile+"/")
		#os.system("rm action1 action2 action3 action4 action5 actionname.txt workflowCreateAction.py")
		action1='action1'
		deleteFile(action1)
		action2='action2'
		deleteFile(action2)
		action3='action3'
		deleteFile(action3)
		action4='action4'
		deleteFile(action4)
		action5='action5'
		deleteFile(action5)
		actionname='actionname.txt'
		deleteFile(actionname)
		deletefile='workflowCreateAction.py'
		deleteFile(deletefile)
		specificTime=(datetime.datetime.now() + datetime.timedelta(hours=0)).strftime('%Y-%m-%d %H:%M')
		
		
		########################################
		logging.info("oozie任务名称："+mission_val)
		logging.info("脚本执行时间："+time_val)
		logging.info("erp名字："+erpName_val)
		logging.info("erp密码："+erpPassword_val)
		with open(allFile+"/time.properties","w") as f:
			f.write("start="+specificTime[0:10]+"T"+specificTime[11:16]+"+0800\n")
			#f.write("start="+startTime)
			f.write("end=2222-02-22T22:22+0800\n")
			f.write("frequency={}".format(time_val))#前端传入
		#os.system("cp "+ooziePorperties+"/time.properties "+allFile+"/")
		os.system("cat "+allFile+"/time.properties >> "+allFile+"/"+jobproperties)
	
	
		####上传配置文件：
		os.system(hadoop_home+"hadoop fs -mkdir -p /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		os.system(hadoop_home+"hadoop fs -rm -r /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/*")
		os.system(hadoop_home+"hadoop fs -put -f "+allFile+"/workflow.xml coordinator.xml *.sh /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		#os.system(hadoop_home+"hadoop fs -put -f  /export/datamarket/OozieScript/lib /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		#os.system(hadoop_home+"hadoop fs -put "+ooziePorperties+"hive-site.xml /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		os.system(hadoop_home+"hadoop fs -ls /user/root/oozie-apps/ceshi/"+today+"/"+allFile.split('/')[-1]+"/")
		os.system(oozie_home+"oozie job -config "+allFile+"/job.properties -run")
	else:
		pass
	



def run(mission_val,oozieShell):
	#mission_val='mission_val'
	#获取oozie任务的相关信息
	db_path = os.path.normpath('/export/datamarket/OozieProperties/oozie_project/db.sqlite3') 
	try: 
		conn = sqlite3.connect(db_path) 
		cursor = conn.cursor()
		sql = "select * from oozie_configuration where mission_val='{}'".format(mission_val) 
		try:
			cursor.execute(sql)
			allresult=cursor.fetchone()
			mission_val=allresult[1]
			time_val=allresult[2]
			erpName_val=allresult[3]
			erpPassword_val=allresult[4]
			# 提交SQL
			conn.commit()
			
		except:
			#发生错误时回滚
			conn.rollback()
			# 关闭数据库连接
		conn.close()
	except Exception as e: 
		print ('------', str(e)) 
		try: 
			conn.close() 
		except Exception as e: 
			pass
	today=datetime.datetime.now().strftime('%Y-%m-%d')
	today='2018-11-10'
	#print("\033[1;34;42m今天日期为："+today+"\033[0m")
	fileList=[]

	##需要执行的文件的目录所在
	os.system("mkdir -p /export/datamarket/OozieExec/"+today+"/"+mission_val)
	os.system("rm /export/datamarket/OozieExec/"+today+"/"+mission_val+"/*.sh")
	os.system("cp "+oozieShell+" /export/datamarket/OozieExec/"+today+"/"+mission_val) 
	os.system("chmod +x /export/datamarket/OozieExec/"+today+"/"+mission_val+"/*.sh") 
	execPath='/export/datamarket/OozieExec/'+today+'/'+mission_val
	
	allfileList=[]
	
	if not os.listdir(execPath):
		print("目录为空")
	else:
		#dirList=os.listdir(execPath)
	
		start = time.time()
		allfileList=[execPath]
		#main(todayfile,today,execPath,mission_val,time_val,erpName_val,erpPassword_val)
		#之前是为了同时提交并自动配置多个任务，所以用进程池
		with ProcessPoolExecutor(max_workers=32) as pool:
			futures =[pool.submit(main, todayfile,today,mission_val,time_val,erpName_val,erpPassword_val) for todayfile in allfileList]
			for future in futures:
				logging.info('\n\n\n\t执行中:%s, 已完成:%s\n\n' % (future.running(), future.done()))
	
			for future in as_completed(futures):
				print ('############################################# 分界线 ######################################################')
				logging.info('\n\n\n\t执行中:%s, 已完成:%s\n\n' % (future.running(), future.done()))
		end = time.time()
	
		logging.info('\n\n\n\t\033[1;34;42m提交任务用时 %.3f seconds.\033[0m\n\n' % (end - start))
		allDir='\n'.join(allfileList)
		logging.info("\n\n\n\033[1;34;42m当前执行目录为：\n"+allDir+"\033[0m\n\n")
